# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

import sqlite3

from scrapy import signals
from scrapy.exceptions import DropItem
from scrapy.exporters import JsonLinesItemExporter

from PhenixScrapy.exporters import Sqlite3ItemExporter


class PhenixscrapyPipeline(object):
    def __init__(self, db):
        self.exporter = Sqlite3ItemExporter(db, ensure_ascii=False) # TODO 解决unicode码转中文问题

    @classmethod
    def from_crawler(cls, crawler):
        pipeline = cls(db=crawler.settings.get('SQLITE_DB'))  # 从 settings.py 提取)
        crawler.signals.connect(pipeline.spider_opened, signals.spider_opened)
        crawler.signals.connect(pipeline.spider_closed, signals.spider_closed)
        return pipeline

    def spider_opened(self, spider):
        self.exporter.start_exporting()

    def spider_closed(self, spider):
        self.exporter.finish_exporting()
        self.exporter.__del__()

    def process_item(self, item, spider):
        if item:
            self.exporter.export_item(item)
        else:
            raise DropItem("Missing %s" % item)
        return item


# 将所有(从所有spider中)爬取到的item，
# 存储到一个独立地 items.json 文件，每行包含一个序列化为JSON格式的item:
class JsonWriterPipeline(object):
    def __init__(self):
        self.file = open('items.json', 'wb')
        self.exporter = JsonLinesItemExporter(self.file, ensure_ascii=False)    # TODO 解决unicode码转中文问题

    @classmethod
    def from_crawler(cls, crawler):
        pipeline = cls()
        crawler.signals.connect(pipeline.spider_opened, signals.spider_opened)
        crawler.signals.connect(pipeline.spider_closed, signals.spider_closed)
        return pipeline

    def spider_opened(self, spider):
        self.exporter.start_exporting()

    def spider_closed(self, spider):
        self.exporter.finish_exporting()

    def process_item(self, item, spider):
        self.exporter.export_item(item)
        return item


class DmozPipeline(object):
    def __init__(self, db):
        self.SQLITE_DB = db

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            db=crawler.settings.get('SQLITE_DB'),  # 从 settings.py 提取
        )

    def open_spider(self, spider):
        self.conn = sqlite3.connect(self.SQLITE_DB)

    def close_spider(self, spider):
        self.conn.close()

    def process_item(self, item, spider):
        if item:
            sql = "INSERT INTO {0}(title, link, description) VALUES(?,?,?)" .format(self.SQLITE_TABLE)
            self.conn.execute(sql, (str(item['title']), str(item['link']), str(item['description'])))
            self.conn.commit()
            print "---DmozPipeline----", type(item['title']), type(item['link']), type(item['description'])
        else:
            raise DropItem("Missing %s" % item)
        return item
